package com.jt.blog.crawler;

import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.fetcher.PageFetcher;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;

import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

/**
 * @author TIM
 * @create 2016-12-10 11
 **/
public class BlogCrawlerController {

    private static Map<Long,Integer> statusMap = Collections.synchronizedMap(new HashMap<Long,Integer>());

    public static void crawler(BlogParserRegEx regEx) throws Exception{
        statusMap.remove(regEx.getUserId());
        BlogParser.getInstance().init(regEx);
        String crawlStorageFolder = "/data/crawl/root";
        int numberOfCrawlers = 1;
        CrawlConfig config = new CrawlConfig();
        config.setCrawlStorageFolder(crawlStorageFolder);
        PageFetcher pageFetcher = new PageFetcher(config);
        RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
        robotstxtConfig.setUserAgentName("tim-spider");
        RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
        CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);
        controller.addSeed(regEx.getSeedUrl());
        controller.start(BlogCrawler.class, numberOfCrawlers);
        statusMap.put(regEx.getUserId(),1);
    }

    public static Integer status(Long userId){
        return statusMap.get(userId);
    }

    public static void setStatus(Long userId,Integer status){
        statusMap.put(userId,status);
    }
}
